{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import (absolute_import, division,\n",
    "                        print_function, unicode_literals)\n",
    "from builtins import *\n",
    "from io import open\n",
    "\n",
    "import time, datetime\n",
    "\n",
    "from NoisyNLP.utils import *\n",
    "from NoisyNLP.features import *\n",
    "from NoisyNLP.models import *\n",
    "from NoisyNLP.experiments import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_files = [\"./data/cleaned/train.BIEOU.tsv\"]\n",
    "dev_files = [\"./data/cleaned/dev.BIEOU.tsv\", \"./data/cleaned/dev_2015.BIEOU.tsv\"]\n",
    "test_files = [\"./data/cleaned/test.BIEOU.tsv\"]\n",
    "vocab_file = \"./vocab.no_extras.txt\"\n",
    "outdir = \"./test_exp\"\n",
    "wordvec_file = \"/home/entity/Downloads/GloVe/glove.twitter.27B.200d.txt.processed.txt\"\n",
    "dictionary_dir=\"./data/cleaned/custom_lexicons/\"\n",
    "gimple_twitter_brown_clusters_dir=\"/home/entity/Downloads/GloVe/50mpaths2\"\n",
    "data_brown_cluster_dir=\"word_clusters/\"\n",
    "data_clark_cluster_dir=\"clark_clusters/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sequences:  7670\n"
     ]
    }
   ],
   "source": [
    "exp = Experiment(outdir, train_files, dev_files, test_files, vocab_file)\n",
    "all_sequences = [[t[0] for t in seq] \n",
    "                        for seq in (exp.train_sequences + exp.dev_sequences + exp.test_sequences)]\n",
    "print(\"Total sequences: \", len(all_sequences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wv_model = WordVectors(all_sequences,wordvec_file)\n",
    "word2vec_clusters = wv_model.get_clusters(n_clusters=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "read dict time.holiday\n",
      "read dict location.country\n",
      "read dict award.award\n",
      "read dict location\n",
      "read dict movies.txt.results.txt\n",
      "read dict lastname.5000\n",
      "read dict sports.sports_team\n",
      "read dict tv.tv_program\n",
      "read dict book.newspaper\n",
      "read dict internet.website\n",
      "read dict base.events.festival_series\n",
      "read dict products.txt.results.txt\n",
      "read dict persons.txt.results.txt\n",
      "read dict tv.tv_network\n",
      "read dict cvg.computer_videogame\n",
      "read dict business.consumer_product\n",
      "read dict people.family_name\n",
      "read dict tvshows.txt.results.txt\n",
      "read dict government.government_agency\n",
      "read dict venues\n",
      "read dict broadcast.tv_channel\n",
      "read dict music_artists.txt.results.txt\n",
      "read dict automotive.make\n",
      "read dict product\n",
      "read dict business.consumer_company\n",
      "read dict music_artists.txt\n",
      "read dict venture_capital.venture_funded_company\n",
      "read dict cvg.cvg_developer\n",
      "read dict architecture.museum\n",
      "read dict lower.5000\n",
      "read dict automotive.model\n",
      "read dict companynames.txt.results.txt\n",
      "read dict musicartist_namevariants.unique.txt\n",
      "read dict time.recurring_event\n",
      "read dict cities.txt.results.txt\n",
      "read dict sports.sports_league\n",
      "read dict firstname.5k\n",
      "read dict business.brand\n",
      "read dict buildings.txt.results.txt\n",
      "read dict musicartist_names.unique.txt\n",
      "read dict english.stop\n",
      "read dict all_geonames.txt\n",
      "read dict education.university\n",
      "read dict cvg.cvg_platform\n",
      "read dict sportsteams.txt.results.txt\n",
      "read dict cap.1000\n",
      "read dict business.sponsor\n",
      "read dict transportation.road\n"
     ]
    }
   ],
   "source": [
    "dict_features = DictionaryFeatures(dictionary_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gimple_brown_cf = ClusterFeatures(gimple_twitter_brown_clusters_dir, cluster_type=\"brown\")\n",
    "gimple_brown_cf.set_cluster_file_path(gimple_twitter_brown_clusters_dir)\n",
    "gimple_brown_clusters = gimple_brown_cf.read_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_brown_cf = ClusterFeatures(data_brown_cluster_dir, cluster_type=\"brown\")\n",
    "data_brown_cf.set_cluster_file_path()\n",
    "data_brown_clusters = data_brown_cf.read_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_clark_cf = ClusterFeatures(data_clark_cluster_dir, cluster_type=\"clark\", n_clusters=32)\n",
    "data_clark_cf.set_cluster_file_path()\n",
    "data_clark_clusters = data_clark_cf.read_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(\"pickled_data/wv_model.pkl\", \"wb+\") as fp:\n",
    "    pickle.dump(wv_model, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"pickled_data/word2vec_clusters.pkl\", \"wb+\") as fp:\n",
    "    pickle.dump(word2vec_clusters, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_exp1(sequences):\n",
    "    X = [sent2features(s, vocab=None,\n",
    "                         dict_features=dict_features, vocab_presence_only=False,\n",
    "                         window=2, interactions=True, dict_interactions=False,\n",
    "                         lowercase=True, dropout=0, word2vec_model=wv_model.model,\n",
    "                        cluster_vocabs=[\n",
    "            gimple_brown_clusters,\n",
    "            data_brown_clusters,\n",
    "            data_clark_clusters\n",
    "        ])\n",
    "         for s in sequences]\n",
    "    y = [sent2labels(s) for s in sequences]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:00:32.735653\n",
      "Dev feature generation took: 0:00:14.997115\n",
      "Test feature generation took: 0:00:36.278326\n",
      "Train: 2394, 2394\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:09:09.681945\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 46469 tokens.\n",
      "Phrases: total=1499, found=1261, correct=1226\n",
      "Overall accuracy: 99.21\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      97.22   81.79  88.84      0.0\n",
      "1       company      98.50   76.61  86.18    133.0\n",
      "2      facility      96.59   80.95  88.08     88.0\n",
      "3       geo-loc      96.12   89.53  92.71    258.0\n",
      "4         movie     100.00   79.41  88.52     27.0\n",
      "5   musicartist     100.00   74.55  85.42     41.0\n",
      "6         other      98.31   77.33  86.57    177.0\n",
      "7        person      96.38   88.86  92.47    414.0\n",
      "8       product     100.00   69.39  81.93     68.0\n",
      "9    sportsteam      94.59   68.63  79.55     37.0\n",
      "10       tvshow     100.00   52.94  69.23     18.0\n",
      "Running:\n",
      "cat \"./test_exp/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 46469 tokens.\n",
      "Phrases: total=1499, found=1261, correct=1241\n",
      "Overall accuracy: 99.26\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.41   82.79  89.93      0.0\n",
      "1               98.41   82.79  89.93   1261.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=594, correct=338\n",
      "Overall accuracy: 94.95\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      56.90   36.07  44.15      0.0\n",
      "1       company      50.00   18.06  26.53     26.0\n",
      "2      facility      52.94   19.57  28.57     17.0\n",
      "3       geo-loc      55.79   65.43  60.23    190.0\n",
      "4         movie      33.33    5.56   9.52      3.0\n",
      "5   musicartist      50.00    1.75   3.39      2.0\n",
      "6         other      33.98   19.55  24.82    103.0\n",
      "7        person      69.30   64.75  66.95    228.0\n",
      "8       product      40.00    8.70  14.29     10.0\n",
      "9    sportsteam      84.62   10.48  18.64     13.0\n",
      "10       tvshow       0.00    0.00   0.00      2.0\n",
      "Running:\n",
      "cat \"./test_exp/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=589, correct=437\n",
      "Overall accuracy: 95.95\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      74.19   46.64  57.27      0.0\n",
      "1               74.19   46.64  57.27    589.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2102, correct=1161\n",
      "Overall accuracy: 92.51\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      55.23   33.28  41.53      0.0\n",
      "1       company      81.91   24.76  38.02    188.0\n",
      "2      facility      44.14   19.37  26.92    111.0\n",
      "3       geo-loc      71.36   65.95  68.55    817.0\n",
      "4         movie       0.00    0.00   0.00      5.0\n",
      "5   musicartist       0.00    0.00   0.00      4.0\n",
      "6         other      26.98   14.38  18.76    315.0\n",
      "7        person      43.68   54.62  48.54    609.0\n",
      "8       product      25.93    2.85   5.13     27.0\n",
      "9    sportsteam      69.57   10.88  18.82     23.0\n",
      "10       tvshow      33.33    3.03   5.56      3.0\n",
      "Running:\n",
      "cat \"./test_exp/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2092, correct=1514\n",
      "Overall accuracy: 94.05\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      72.37   43.39  54.26      0.0\n",
      "1               72.37   43.39  54.26   2092.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 25 likely transitions:\n",
      "O      -> O       4.644402\n",
      "B-person -> E-person 4.411072\n",
      "I-other -> E-other 4.175164\n",
      "I-other -> I-other 4.124702\n",
      "B-other -> I-other 3.928763\n",
      "B-other -> E-other 3.731435\n",
      "B-facility -> E-facility 3.701140\n",
      "B-product -> E-product 3.352413\n",
      "I-facility -> E-facility 3.131877\n",
      "B-musicartist -> E-musicartist 3.066101\n",
      "B-company -> E-company 2.957187\n",
      "B-facility -> I-facility 2.954348\n",
      "B-geo-loc -> E-geo-loc 2.939255\n",
      "I-product -> I-product 2.675727\n",
      "B-sportsteam -> E-sportsteam 2.524979\n",
      "B-movie -> E-movie 2.385254\n",
      "B-tvshow -> E-tvshow 2.330000\n",
      "I-product -> E-product 2.308091\n",
      "B-product -> I-product 2.251306\n",
      "I-person -> E-person 2.223817\n",
      "U-person -> O       2.219954\n",
      "O      -> U-person 2.197314\n",
      "B-movie -> I-movie 2.187085\n",
      "I-movie -> E-movie 2.111887\n",
      "E-person -> O       2.062839\n",
      "3.155129 O        __EOS__\n",
      "2.953221 O        __BOS__\n",
      "2.699114 U-geo-loc __BROWN_CLUSTER_0__:11100110101\n",
      "1.955702 U-person __BROWN_CLUSTER_0__:111001110\n",
      "1.927359 U-geo-loc __BROWN_CLUSTER_0__:11100110100\n",
      "1.818587 B-facility __BROWN_CLUSTER_0__[-1]:10111100\n",
      "1.745969 U-company __BROWN_CLUSTER_0__:111001100001\n",
      "1.710580 U-facility __BROWN_CLUSTER_0__[-1]:10111100\n",
      "1.611846 U-sportsteam __BROWN_CLUSTER_0__:1111011010\n",
      "1.565234 U-company __BROWN_CLUSTER_0__:111101011000\n"
     ]
    }
   ],
   "source": [
    "exp.describe_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train + Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Directory ./test_exp_train_dev doesn't exist.\n",
      "Directory ./test_exp_train_dev created.\n"
     ]
    }
   ],
   "source": [
    "outdir = \"./test_exp_train_dev\"\n",
    "exp = Experiment(outdir, train_files + dev_files, dev_files, test_files, vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:00:41.075128\n",
      "Dev feature generation took: 0:00:13.437942\n",
      "Test feature generation took: 0:00:34.648778\n",
      "Train: 3814, 3814\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:15:40.636907\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2033, correct=1949\n",
      "Overall accuracy: 99.10\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      95.87   80.01  87.22      0.0\n",
      "1       company      97.30   74.07  84.11    185.0\n",
      "2      facility      95.04   76.16  84.56    121.0\n",
      "3       geo-loc      93.38   86.79  89.96    408.0\n",
      "4         movie     100.00   80.77  89.36     42.0\n",
      "5   musicartist      96.51   74.11  83.84     86.0\n",
      "6         other      96.56   76.49  85.36    320.0\n",
      "7        person      95.62   88.17  91.74    639.0\n",
      "8       product      98.04   69.44  81.30    102.0\n",
      "9    sportsteam      98.18   69.23  81.20    110.0\n",
      "10       tvshow     100.00   47.62  64.52     20.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2033, correct=1991\n",
      "Overall accuracy: 99.18\n",
      "  category  precision  recall    F1  support\n",
      "0  overall      97.93   81.73  89.1      0.0\n",
      "1               97.93   81.73  89.1   2033.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=782, correct=744\n",
      "Overall accuracy: 99.01\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      95.14   79.40  86.56      0.0\n",
      "1       company      98.04   69.44  81.30     51.0\n",
      "2      facility      94.12   69.57  80.00     34.0\n",
      "3       geo-loc      87.65   87.65  87.65    162.0\n",
      "4         movie     100.00   83.33  90.91     15.0\n",
      "5   musicartist      95.45   73.68  83.17     44.0\n",
      "6         other      97.22   78.21  86.69    144.0\n",
      "7        person      96.85   88.11  92.27    222.0\n",
      "8       product      94.59   76.09  84.34     37.0\n",
      "9    sportsteam     100.00   67.62  80.68     71.0\n",
      "10       tvshow     100.00   25.00  40.00      2.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=782, correct=765\n",
      "Overall accuracy: 99.11\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      97.83   81.64  89.01      0.0\n",
      "1               97.83   81.64  89.01    782.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2224, correct=1324\n",
      "Overall accuracy: 92.99\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      59.53   37.95  46.35      0.0\n",
      "1       company      76.05   29.10  42.09    238.0\n",
      "2      facility      53.68   28.85  37.53    136.0\n",
      "3       geo-loc      73.60   66.86  70.07    803.0\n",
      "4         movie       0.00    0.00   0.00      0.0\n",
      "5   musicartist      42.11    4.17   7.58     19.0\n",
      "6         other      45.14   24.37  31.65    319.0\n",
      "7        person      46.86   56.67  51.30    589.0\n",
      "8       product      40.00    5.69   9.96     35.0\n",
      "9    sportsteam      43.37   24.49  31.30     83.0\n",
      "10       tvshow      50.00    3.03   5.71      2.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2219, correct=1635\n",
      "Overall accuracy: 94.35\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      73.68   46.86  57.29      0.0\n",
      "1               73.68   46.86  57.29   2219.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 25 likely transitions:\n",
      "B-person -> E-person 5.003097\n",
      "O      -> O       4.906769\n",
      "I-other -> E-other 4.725323\n",
      "I-other -> I-other 4.586412\n",
      "B-other -> I-other 4.405346\n",
      "B-other -> E-other 4.210546\n",
      "I-product -> I-product 4.137379\n",
      "B-facility -> E-facility 3.962764\n",
      "B-product -> E-product 3.861498\n",
      "I-facility -> E-facility 3.755069\n",
      "B-facility -> I-facility 3.638113\n",
      "B-company -> E-company 3.637893\n",
      "I-product -> E-product 3.546175\n",
      "B-musicartist -> E-musicartist 3.532204\n",
      "B-sportsteam -> E-sportsteam 3.431834\n",
      "B-geo-loc -> E-geo-loc 3.311323\n",
      "B-product -> I-product 3.293329\n",
      "B-movie -> E-movie 3.077283\n",
      "I-person -> E-person 2.893064\n",
      "B-tvshow -> E-tvshow 2.783769\n",
      "I-musicartist -> E-musicartist 2.735140\n",
      "B-company -> I-company 2.685208\n",
      "I-geo-loc -> E-geo-loc 2.682392\n",
      "B-musicartist -> I-musicartist 2.669835\n",
      "O      -> U-person 2.583387\n",
      "3.628546 O        __EOS__\n",
      "3.055463 O        __BOS__\n",
      "2.542522 U-sportsteam __BROWN_CLUSTER_0__:1111011010\n",
      "2.536144 U-geo-loc __BROWN_CLUSTER_0__:11100110101\n",
      "2.019569 U-person __BROWN_CLUSTER_0__:111001110\n",
      "2.014693 U-geo-loc __BROWN_CLUSTER_0__:11100110100\n",
      "2.004246 B-facility __BROWN_CLUSTER_0__[-1]:10111100\n",
      "1.826677 U-facility __BROWN_CLUSTER_0__[-1]:10111100\n",
      "1.825996 U-company __BROWN_CLUSTER_0__:111001100001\n",
      "1.807465 U-product __BROWN_CLUSTER_0__:11110101101110\n"
     ]
    }
   ],
   "source": [
    "exp.describe_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word vectors and resources enriched by test sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_enriched_data_brown_cluster_dir=\"test_enriched/brown_clusters/\"\n",
    "test_enriched_data_clark_cluster_dir=\"test_enriched/clark_clusters/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "brown_exec_path=\"/home/entity/Downloads/brown-cluster/wcluster\"\n",
    "brown_input_data_path=\"test_enriched/all_sequences.brown.txt\"\n",
    "test_enriched_data_brown_cf = ClusterFeatures(test_enriched_data_brown_cluster_dir,\n",
    "                                              cluster_type=\"brown\", n_clusters=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_enriched_data_brown_cf.set_cluster_file_path()\n",
    "test_enriched_data_brown_clusters = test_enriched_data_brown_cf.read_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_enriched_data_clark_cf = ClusterFeatures(test_enriched_data_clark_cluster_dir,\n",
    "                                              cluster_type=\"clark\", n_clusters=32)\n",
    "test_enriched_data_clark_cf.set_cluster_file_path()\n",
    "test_enriched_data_clark_clusters = test_enriched_data_clark_cf.read_clusters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_exp2(sequences):\n",
    "    X = [sent2features(s, vocab=None,\n",
    "                         dict_features=dict_features, vocab_presence_only=False,\n",
    "                         window=2, interactions=True, dict_interactions=False,\n",
    "                         lowercase=True, dropout=0, word2vec_model=wv_model.model,\n",
    "                        cluster_vocabs=[\n",
    "            gimple_brown_clusters,\n",
    "            test_enriched_data_brown_clusters,\n",
    "            test_enriched_data_clark_clusters\n",
    "        ])\n",
    "         for s in sequences]\n",
    "    y = [sent2labels(s) for s in sequences]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "outdir = \"./test_exp_train_dev_test_enriched\"\n",
    "exp = Experiment(outdir, train_files + dev_files, dev_files, test_files, vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:00:40.592333\n",
      "Dev feature generation took: 0:00:13.204734\n",
      "Test feature generation took: 0:00:34.990218\n",
      "Train: 3814, 3814\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:14:49.150418\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2125, correct=2076\n",
      "Overall accuracy: 99.36\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      97.69   85.22  91.03      0.0\n",
      "1       company      98.98   79.84  88.38    196.0\n",
      "2      facility      96.85   81.46  88.49    127.0\n",
      "3       geo-loc      96.87   91.57  94.15    415.0\n",
      "4         movie     100.00   82.69  90.53     43.0\n",
      "5   musicartist      97.75   77.68  86.57     89.0\n",
      "6         other      97.07   81.93  88.86    341.0\n",
      "7        person      97.54   91.63  94.49    651.0\n",
      "8       product      99.12   77.78  87.16    113.0\n",
      "9    sportsteam      99.18   77.56  87.05    122.0\n",
      "10       tvshow     100.00   66.67  80.00     28.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2125, correct=2100\n",
      "Overall accuracy: 99.40\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.82   86.21  92.09      0.0\n",
      "1               98.82   86.21  92.09   2125.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=813, correct=792\n",
      "Overall accuracy: 99.28\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      97.42   84.53  90.51      0.0\n",
      "1       company      96.61   79.17  87.02     59.0\n",
      "2      facility      97.30   78.26  86.75     37.0\n",
      "3       geo-loc      94.12   88.89  91.43    153.0\n",
      "4         movie     100.00   83.33  90.91     15.0\n",
      "5   musicartist      95.56   75.44  84.31     45.0\n",
      "6         other      99.33   82.68  90.24    149.0\n",
      "7        person      97.80   90.98  94.27    227.0\n",
      "8       product      97.50   84.78  90.70     40.0\n",
      "9    sportsteam     100.00   79.05  88.30     83.0\n",
      "10       tvshow     100.00   62.50  76.92      5.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=813, correct=804\n",
      "Overall accuracy: 99.35\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.89   85.81  91.89      0.0\n",
      "1               98.89   85.81  91.89    813.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2260, correct=1360\n",
      "Overall accuracy: 93.00\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      60.18   38.98  47.31      0.0\n",
      "1       company      75.27   33.28  46.15    275.0\n",
      "2      facility      53.72   25.69  34.76    121.0\n",
      "3       geo-loc      73.54   68.55  70.96    824.0\n",
      "4         movie       0.00    0.00   0.00      1.0\n",
      "5   musicartist      40.00    3.12   5.80     15.0\n",
      "6         other      44.64   25.38  32.36    336.0\n",
      "7        person      48.50   56.47  52.18    567.0\n",
      "8       product      37.14    5.28   9.25     35.0\n",
      "9    sportsteam      44.05   25.17  32.03     84.0\n",
      "10       tvshow      50.00    3.03   5.71      2.0\n",
      "Running:\n",
      "cat \"./test_exp_train_dev_test_enriched/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2250, correct=1693\n",
      "Overall accuracy: 94.44\n",
      "  category  precision  recall    F1  support\n",
      "0  overall      75.24   48.52  59.0      0.0\n",
      "1               75.24   48.52  59.0   2250.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dict interactions and larger window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_exp3(sequences):\n",
    "    X = [sent2features(s, vocab=None,\n",
    "                         dict_features=dict_features, vocab_presence_only=False,\n",
    "                         window=4, interactions=True, dict_interactions=True,\n",
    "                         lowercase=True, dropout=0, word2vec_model=wv_model.model,\n",
    "                        cluster_vocabs=[\n",
    "            gimple_brown_clusters,\n",
    "            test_enriched_data_brown_clusters,\n",
    "            test_enriched_data_clark_clusters\n",
    "        ])\n",
    "         for s in sequences]\n",
    "    y = [sent2labels(s) for s in sequences]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Directory ./test_exp_tdt_enriched_win4_dict_intract doesn't exist.\n",
      "Directory ./test_exp_tdt_enriched_win4_dict_intract created.\n"
     ]
    }
   ],
   "source": [
    "outdir = \"./test_exp_tdt_enriched_win4_dict_intract\"\n",
    "exp = Experiment(outdir, train_files + dev_files, dev_files, test_files, vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:00:40.270651\n",
      "Dev feature generation took: 0:00:13.110668\n",
      "Test feature generation took: 0:00:33.877503\n",
      "Train: 3814, 3814\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:15:07.623600\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2125, correct=2076\n",
      "Overall accuracy: 99.36\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      97.69   85.22  91.03      0.0\n",
      "1       company      98.98   79.84  88.38    196.0\n",
      "2      facility      96.85   81.46  88.49    127.0\n",
      "3       geo-loc      96.87   91.57  94.15    415.0\n",
      "4         movie     100.00   82.69  90.53     43.0\n",
      "5   musicartist      97.75   77.68  86.57     89.0\n",
      "6         other      97.07   81.93  88.86    341.0\n",
      "7        person      97.54   91.63  94.49    651.0\n",
      "8       product      99.12   77.78  87.16    113.0\n",
      "9    sportsteam      99.18   77.56  87.05    122.0\n",
      "10       tvshow     100.00   66.67  80.00     28.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2125, correct=2100\n",
      "Overall accuracy: 99.40\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.82   86.21  92.09      0.0\n",
      "1               98.82   86.21  92.09   2125.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=813, correct=792\n",
      "Overall accuracy: 99.28\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      97.42   84.53  90.51      0.0\n",
      "1       company      96.61   79.17  87.02     59.0\n",
      "2      facility      97.30   78.26  86.75     37.0\n",
      "3       geo-loc      94.12   88.89  91.43    153.0\n",
      "4         movie     100.00   83.33  90.91     15.0\n",
      "5   musicartist      95.56   75.44  84.31     45.0\n",
      "6         other      99.33   82.68  90.24    149.0\n",
      "7        person      97.80   90.98  94.27    227.0\n",
      "8       product      97.50   84.78  90.70     40.0\n",
      "9    sportsteam     100.00   79.05  88.30     83.0\n",
      "10       tvshow     100.00   62.50  76.92      5.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=813, correct=804\n",
      "Overall accuracy: 99.35\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.89   85.81  91.89      0.0\n",
      "1               98.89   85.81  91.89    813.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2260, correct=1360\n",
      "Overall accuracy: 93.00\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      60.18   38.98  47.31      0.0\n",
      "1       company      75.27   33.28  46.15    275.0\n",
      "2      facility      53.72   25.69  34.76    121.0\n",
      "3       geo-loc      73.54   68.55  70.96    824.0\n",
      "4         movie       0.00    0.00   0.00      1.0\n",
      "5   musicartist      40.00    3.12   5.80     15.0\n",
      "6         other      44.64   25.38  32.36    336.0\n",
      "7        person      48.50   56.47  52.18    567.0\n",
      "8       product      37.14    5.28   9.25     35.0\n",
      "9    sportsteam      44.05   25.17  32.03     84.0\n",
      "10       tvshow      50.00    3.03   5.71      2.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_win4_dict_intract/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2250, correct=1693\n",
      "Overall accuracy: 94.44\n",
      "  category  precision  recall    F1  support\n",
      "0  overall      75.24   48.52  59.0      0.0\n",
      "1               75.24   48.52  59.0   2250.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## With word features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_exp4(sequences):\n",
    "    X = [sent2features(s, vocab=None,\n",
    "                         dict_features=dict_features, vocab_presence_only=False,\n",
    "                         window=4, interactions=True, dict_interactions=True,\n",
    "                         lowercase=False, dropout=0, word2vec_model=wv_model.model,\n",
    "                        cluster_vocabs=[\n",
    "            gimple_brown_clusters,\n",
    "            test_enriched_data_brown_clusters,\n",
    "            test_enriched_data_clark_clusters\n",
    "        ])\n",
    "         for s in sequences]\n",
    "    y = [sent2labels(s) for s in sequences]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Directory ./test_exp_tdt_enriched_word_Features doesn't exist.\n",
      "Directory ./test_exp_tdt_enriched_word_Features created.\n"
     ]
    }
   ],
   "source": [
    "outdir = \"./test_exp_tdt_enriched_word_Features\"\n",
    "exp = Experiment(outdir, train_files + dev_files, dev_files, test_files, vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:00:40.389356\n",
      "Dev feature generation took: 0:00:12.946482\n",
      "Test feature generation took: 0:00:34.710360\n",
      "Train: 3814, 3814\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:16:02.068697\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2167, correct=2129\n",
      "Overall accuracy: 99.45\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      98.25   87.40  92.50      0.0\n",
      "1       company      98.53   82.72  89.93    204.0\n",
      "2      facility      96.92   83.44  89.68    130.0\n",
      "3       geo-loc      97.37   92.94  95.10    419.0\n",
      "4         movie     100.00   86.54  92.78     45.0\n",
      "5   musicartist      97.87   82.14  89.32     94.0\n",
      "6         other      97.97   83.66  90.25    345.0\n",
      "7        person      98.63   93.22  95.85    655.0\n",
      "8       product      99.14   79.86  88.46    116.0\n",
      "9    sportsteam      99.22   81.41  89.44    128.0\n",
      "10       tvshow     100.00   73.81  84.93     31.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2167, correct=2145\n",
      "Overall accuracy: 99.48\n",
      "  category  precision  recall    F1  support\n",
      "0  overall      98.98   88.05  93.2      0.0\n",
      "1               98.98   88.05  93.2   2167.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=825, correct=809\n",
      "Overall accuracy: 99.37\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      98.06   86.34  91.83      0.0\n",
      "1       company      95.00   79.17  86.36     60.0\n",
      "2      facility      97.37   80.43  88.10     38.0\n",
      "3       geo-loc      94.81   90.12  92.41    154.0\n",
      "4         movie     100.00   88.89  94.12     16.0\n",
      "5   musicartist      96.00   84.21  89.72     50.0\n",
      "6         other     100.00   83.80  91.19    150.0\n",
      "7        person      99.55   90.98  95.07    223.0\n",
      "8       product      97.56   86.96  91.95     41.0\n",
      "9    sportsteam     100.00   82.86  90.62     87.0\n",
      "10       tvshow     100.00   75.00  85.71      6.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=825, correct=816\n",
      "Overall accuracy: 99.41\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      98.91   87.09  92.62      0.0\n",
      "1               98.91   87.09  92.62    825.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2262, correct=1364\n",
      "Overall accuracy: 93.00\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      60.30   39.09  47.44      0.0\n",
      "1       company      76.53   34.08  47.16    277.0\n",
      "2      facility      52.99   24.51  33.51    117.0\n",
      "3       geo-loc      73.49   69.00  71.18    830.0\n",
      "4         movie       0.00    0.00   0.00      1.0\n",
      "5   musicartist      42.86    3.12   5.83     14.0\n",
      "6         other      43.66   25.04  31.83    339.0\n",
      "7        person      48.59   56.47  52.23    566.0\n",
      "8       product      38.24    5.28   9.29     34.0\n",
      "9    sportsteam      45.12   25.17  32.31     82.0\n",
      "10       tvshow      50.00    3.03   5.71      2.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2252, correct=1695\n",
      "Overall accuracy: 94.45\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      75.27   48.58  59.05      0.0\n",
      "1               75.27   48.58  59.05   2252.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## With Global features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%aimport NoisyNLP.experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'product',\n",
       " u'facility',\n",
       " u'movie',\n",
       " u'company',\n",
       " u'sportsteam',\n",
       " u'musicartist',\n",
       " u'person',\n",
       " u'other',\n",
       " u'geo-loc',\n",
       " u'tvshow']"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outdir = \"./test_exp_tdt_enriched_word_Features_global\"\n",
    "exp = Experiment(outdir, train_files + dev_files, dev_files, test_files, vocab_file)\n",
    "cat_names = list(set([t[1][2:] for seq in exp.train_sequences for t in seq if t[1] != \"O\"]))\n",
    "cat_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "global_features = GlobalFeatures(word2vec_model=wv_model.model, cluster_vocabs=gimple_brown_clusters, cat_names=cat_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[Tag(token=u'@SammieLynnsMom', tag=u'O'),\n",
       "  Tag(token=u'@tg10781', tag=u'O'),\n",
       "  Tag(token=u'they', tag=u'O'),\n",
       "  Tag(token=u'will', tag=u'O'),\n",
       "  Tag(token=u'be', tag=u'O'),\n",
       "  Tag(token=u'all', tag=u'O'),\n",
       "  Tag(token=u'done', tag=u'O'),\n",
       "  Tag(token=u'by', tag=u'O'),\n",
       "  Tag(token=u'Sunday', tag=u'O'),\n",
       "  Tag(token=u'trust', tag=u'O'),\n",
       "  Tag(token=u'me', tag=u'O'),\n",
       "  Tag(token=u'*wink*', tag=u'O')]]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.train_sequences[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing: product\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1377\n",
      "          1       1.00      0.95      0.98        43\n",
      "\n",
      "avg / total       1.00      1.00      1.00      1420\n",
      "\n",
      "Processing: facility\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1375\n",
      "          1       0.97      0.87      0.92        45\n",
      "\n",
      "avg / total       0.99      1.00      0.99      1420\n",
      "\n",
      "Processing: movie\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1406\n",
      "          1       1.00      0.93      0.96        14\n",
      "\n",
      "avg / total       1.00      1.00      1.00      1420\n",
      "\n",
      "Processing: company\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      1360\n",
      "          1       0.98      0.75      0.85        60\n",
      "\n",
      "avg / total       0.99      0.99      0.99      1420\n",
      "\n",
      "Processing: sportsteam\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1360\n",
      "          1       1.00      0.95      0.97        60\n",
      "\n",
      "avg / total       1.00      1.00      1.00      1420\n",
      "\n",
      "Processing: musicartist\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      1.00      1383\n",
      "          1       0.97      0.76      0.85        37\n",
      "\n",
      "avg / total       0.99      0.99      0.99      1420\n",
      "\n",
      "Processing: person\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      0.99      0.99      1231\n",
      "          1       0.95      0.87      0.91       189\n",
      "\n",
      "avg / total       0.98      0.98      0.98      1420\n",
      "\n",
      "Processing: other\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.97      1.00      0.98      1255\n",
      "          1       0.98      0.78      0.87       165\n",
      "\n",
      "avg / total       0.97      0.97      0.97      1420\n",
      "\n",
      "Processing: geo-loc\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      1297\n",
      "          1       0.96      0.87      0.91       123\n",
      "\n",
      "avg / total       0.99      0.99      0.99      1420\n",
      "\n",
      "Processing: tvshow\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1412\n",
      "          1       1.00      0.75      0.86         8\n",
      "\n",
      "avg / total       1.00      1.00      1.00      1420\n",
      "\n"
     ]
    }
   ],
   "source": [
    "global_features.fit_model(exp.train_sequences, test_sequences=exp.dev_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_X_y_exp5(sequences):\n",
    "    preds = global_features.get_global_predictions(sequences)\n",
    "    X = [sent2features(s, vocab=None,\n",
    "                         dict_features=dict_features, vocab_presence_only=False,\n",
    "                         window=4, interactions=True, dict_interactions=True,\n",
    "                         lowercase=False, dropout=0, word2vec_model=wv_model.model,\n",
    "                        cluster_vocabs=[\n",
    "            gimple_brown_clusters,\n",
    "            test_enriched_data_brown_clusters,\n",
    "            test_enriched_data_clark_clusters\n",
    "        ],\n",
    "                       extra_features=global_features.get_global_sequence_features(s, predictions=p))\n",
    "         for s,p in zip(sequences, preds)]\n",
    "    y = [sent2labels(s) for s in sequences]\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train feature generation took: 0:01:25.250826\n",
      "Dev feature generation took: 0:00:27.527947\n",
      "Test feature generation took: 0:01:13.002616\n",
      "Train: 3814, 3814\n",
      "Dev: 1420, 1420\n",
      "Test: 3856, 3856\n"
     ]
    }
   ],
   "source": [
    "exp.gen_model_data(proc_func=get_X_y_exp5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model fitting took: 0:26:30.177333\n",
      "Evaluating train data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/train.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2295, correct=2273\n",
      "Overall accuracy: 99.71\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      99.04   93.31  96.09      0.0\n",
      "1       company      98.68   92.59  95.54    228.0\n",
      "2      facility      98.55   90.07  94.12    138.0\n",
      "3       geo-loc      98.36   95.90  97.12    428.0\n",
      "4         movie     100.00   96.15  98.04     50.0\n",
      "5   musicartist      98.11   92.86  95.41    106.0\n",
      "6         other      99.46   90.35  94.68    367.0\n",
      "7        person      99.55   95.38  97.42    664.0\n",
      "8       product      98.48   90.28  94.20    132.0\n",
      "9    sportsteam      99.33   94.87  97.05    149.0\n",
      "10       tvshow     100.00   78.57  88.00     33.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/train.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 69519 tokens.\n",
      "Phrases: total=2436, found=2295, correct=2280\n",
      "Overall accuracy: 99.72\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      99.35    93.6  96.39      0.0\n",
      "1               99.35    93.6  96.39   2295.0\n",
      "Evaluating dev data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/dev.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=875, correct=865\n",
      "Overall accuracy: 99.65\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      98.86   92.32  95.47      0.0\n",
      "1       company      97.10   93.06  95.04     69.0\n",
      "2      facility      97.37   80.43  88.10     38.0\n",
      "3       geo-loc      98.04   92.59  95.24    153.0\n",
      "4         movie     100.00   94.44  97.14     17.0\n",
      "5   musicartist      96.36   92.98  94.64     55.0\n",
      "6         other     100.00   91.06  95.32    163.0\n",
      "7        person      99.57   94.26  96.84    231.0\n",
      "8       product      97.67   91.30  94.38     43.0\n",
      "9    sportsteam     100.00   95.24  97.56    100.0\n",
      "10       tvshow     100.00   75.00  85.71      6.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/dev.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 23050 tokens.\n",
      "Phrases: total=937, found=875, correct=869\n",
      "Overall accuracy: 99.67\n",
      "  category  precision  recall     F1  support\n",
      "0  overall      99.31   92.74  95.92      0.0\n",
      "1               99.31   92.74  95.92    875.0\n",
      "Evaluating test data\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/test.bieou.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2305, correct=1346\n",
      "Overall accuracy: 92.87\n",
      "       category  precision  recall     F1  support\n",
      "0       overall      58.39   38.58  46.46      0.0\n",
      "1       company      68.77   36.82  47.96    333.0\n",
      "2      facility      54.78   24.90  34.24    115.0\n",
      "3       geo-loc      71.46   65.72  68.47    813.0\n",
      "4         movie       0.00    0.00   0.00      2.0\n",
      "5   musicartist      71.43    7.81  14.08     21.0\n",
      "6         other      40.24   23.01  29.28    338.0\n",
      "7        person      48.94   56.88  52.61    566.0\n",
      "8       product      27.91    4.88   8.30     43.0\n",
      "9    sportsteam      44.59   22.45  29.86     74.0\n",
      "10       tvshow       0.00    0.00   0.00      0.0\n",
      "Running:\n",
      "cat \"./test_exp_tdt_enriched_word_Features_global/test.notypes.tsv\" | tr '\\t' ' ' | perl -ne '{chomp;s/\\r//g;print $_,\"\\n\";}' | python NoisyNLP/conlleval.py\n",
      "Processed 61908 tokens.\n",
      "Phrases: total=3489, found=2298, correct=1650\n",
      "Overall accuracy: 94.20\n",
      "  category  precision  recall     F1  support\n",
      "0  overall       71.8   47.29  57.02      0.0\n",
      "1                71.8   47.29  57.02   2298.0\n"
     ]
    }
   ],
   "source": [
    "exp.fit_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
